from IPython.display import Image
Image(filename='imgs/banner.png')

%load_ext pretty_jupyter

import pendulum

%%html

<style>
    #Styling {
        font-weight: bold;
        font-family: Helvetica;
    }
</style>

# dependencies
import sys
import re
import pandas as pd
from ipywidgets import HTML

sys.path.append(".")
import Summary
import Relative_risk
import Chi_square

# support methods
def format_count(v):
    return "{:,}".format(v)


def format_prop(prop, decn=1, asperc=True):
    if asperc: prop = prop*100
    return "{}%".format(round(prop, decn))


def formatprop_num(num, den, aspercent, dec):
    if num == 0: return 0
    v = num/den
    if not aspercent: out = round(v, dec)
    else: out = round(v * 100, dec)
    return out


def formatprop_str(num, den, aspercent, dec):
    if num == 0: return '0'
    v = num/den
    if not aspercent: out = f"%.{dec}f" % v
    else: out = f"%.{dec}f" % (v*100)
    if int(float(out)) == 0: return 'less than 1'
    return out


def formatprop(num, den, numeric, aspercent=False, dec=1):
    assert (den >= num)
    if numeric: out = formatprop_num(num=num, den=den, aspercent=aspercent, dec=dec)
    else: out = formatprop_str(num=num, den=den, aspercent=aspercent, dec=dec)
    return out


def report_fields(df, idcol, cols, fillval='NO DATA'):
    data = df[[idcol] + cols].drop_duplicates()
    if fillval:
        count = data[cols].fillna(fillval).value_counts().to_frame().reset_index()
        perc = data[cols].fillna(fillval).value_counts(normalize=True).to_frame().reset_index(
            ).rename(columns={'proportion': 'percent'})
    else:
        count = data[cols].value_counts().to_frame().reset_index()
        perc = data[cols].value_counts(normalize=True).to_frame().reset_index(
            ).rename(columns={'proportion': 'percent'})
    count['count'] = count['count'].apply(lambda x: f"{x:,}")
    perc.percent = perc.percent.apply(lambda x: f"{x*100:.2f}%")
    out = pd.merge(count, perc, on=cols)
    return out


def get_census():
    # county numbers from USCB DHC https://data.census.gov/table/DECENNIALDHC2020.P9?t=Race%20and%20Ethnicity&g=050XX00US06075
    magic = {}
    magic['county'] = {
        'total': {'n': 873965},
        'Black': {'n': 45071}, # 'Black or African American alone'
        'White': {'n': 341306}, # 'White alone'
        'Latine': {'n': 136761}, # 'Hispanic or Latino'
        'Asian': {'n': 294220 + 3244 + 1570}, # 'Asian alone' + 'Native Hawaiian and Other Pacific Islander alone' + 'American Indian and Alaska Native alone'
        'Other/Unknown': {'n': 6347 + 45446}, # 'Some Other Race alone' + 'Population of two or more races'
    }
    realgroups = [k for k in magic['county'].keys() if k != 'total']
    assert magic['county']['total']['n'] == sum([magic['county'][group]['n'] for group in realgroups]), f"\
    Census counts by race group should add up to the total population count.\
    Found {magic['county']['total']['n']} for total population and {sum([magic['county'][group]['n'] for group in realgroups])} group total."
    for race_group in realgroups:
        magic['county'][race_group]['prop'] = formatprop(
            num=magic['county'][race_group]['n'],
            den=magic['county']['total']['n'],
            numeric=True, aspercent=False, dec=5)
    for race_group in realgroups:
        magic['county'][race_group]['perc'] = formatprop(
            num=magic['county'][race_group]['n'],
            den=magic['county']['total']['n'],
            numeric=False, aspercent=True, dec=1)
    return magic

# main
"""This is the processed version of publicly available data requested by the ACLU from the San Francisco District Attorney's Office ("SFDA") covering 2015-2022. Note that the SFDA did not necessarily produce the data themselves and it may also reflect SFPD or SF Sheriff entries."""
sfda = pd.read_parquet("sfda.parquet")
magic = get_census()

# support for breaking down data
cols = {
    'meta': [
        'source', 'filename', 'sheet', 'incident_number', 'court_number',],
    'person': [
        'age_at_arrest', 'gender', 'race', 'ethnicity', 'ethnicity_group',],
    'booking': [
        'incident_number', 'arrest_date', 'age_at_arrest',
        'booked_case_type', 'booked_charge_list',],
    'filing': [
        'court_number', 'filing_date',
        'filed_case_type', 'filed_charge_list', 'description',
        'case_dispo_date', 'case_dispo', 'dispo_description', 'dispo_description_group',
        'status_ctnum', 'status_ctnum_agg', 'status_ctnum_group',],
}
grouped = [c for collist in cols.values() for c in collist]
lost = [c for c in sfda.columns if c not in grouped]
assert not any(lost), f"\
previously had grouped all available columns, however {
lost} have not been grouped."

assert sfda.filename.str.contains('San Francisco County').all()

# add case-based indicators
sfda['case_filed'] = sfda.court_number.notna()
sfda['conviction_any'] = sfda.dispo_description_group.isin((
    'imprisonment', 'jail_probation', 'other_conviction'))

# add charge-based indicators
pcs = {
    148: '148[A-Z]*|148\\.',
    211: '211',
}
for code, patt in pcs.items():
    for stage in ('booked', 'filed'): # note lack of a `convicted_charge_list` field
        chargecol = f'{stage}_charge_list'
        sfda[f'pc_{code}_{stage}'] = sfda[chargecol].str.contains(patt, na=False, flags=re.I)

sfda['booked_148_only'] = sfda.pc_148_booked & sfda.booked_charge_list.apply(
    lambda x: len(x.split(',')) == 1 if x else False)

LABELS = {
    'ethnicity_group': 'someone with a recorded race/ethnicity of',
    'def_black': 'a Black defendant',
    'def_white': 'a White defendant',
    'case_filed': 'a case against a defendant',
    'conviction_any': 'conviction on at least one charge', # the data are not more specific than this
    'pc_148_booked': 'a booked charge for resisting arrest',
    'booked_148_only': 'arrest(s) solely for resisting arrest',
    'pc_148_filed': 'a filed charge for resisting arrest',
    'pc_211_booked': 'a booked charge for robbery',
    'pc_211_filed': 'a filed charge for robbery',
}

Summary.Summary.__init__

Summary.PARAMS

pc148_sum = Summary.Summary(
    df=sfda,
    params={
        'INDICATOR_COL': 'booked_148_only',
        'INDICATOR_OP': 'are of',
        'GROUP_COL': 'ethnicity_group',
        'RENAMER': {
            'ethnicity_group': 'Race/Ethnicity',
            True: 'PC 148 alone',
            False: 'Other charge(s)'
        }
    },
    labels=LABELS,
)

pc148_sum_gc = pc148_sum.getmagic()['GROUP_COUNTS'].copy()

pc148_sum.getmagic()

Relative_risk.Contingency.__init__

Relative_risk.CONTINGENCY_PARAMS

Relative_risk.RATIO_PARAMS

sfda['def_black'] = sfda.ethnicity_group == 'Black'
sfda['def_white'] = sfda.ethnicity_group == 'White'
sfda['def_black_or_hispanic'] = sfda.ethnicity_group.isin(('Black', 'Latine'))
sfda['def_not_black'] = ~sfda.def_black

# setup comparison groups based on race data
sfda.loc[(sfda.def_black) | (sfda.def_white), 'comparison_group_1'] = sfda.ethnicity_group
sfda.loc[sfda.def_black, 'comparison_group_2'] = sfda.ethnicity_group
sfda.loc[sfda.def_not_black, 'comparison_group_2'] = 'Race other than Black'
sfda.loc[sfda.def_white, 'comparison_group_3'] = sfda.ethnicity_group
sfda.loc[sfda.def_black_or_hispanic, 'comparison_group_3'] = 'Black or Hispanic'

pc148_rr_tbl = Relative_risk.Contingency(
    df=sfda,
    params={
        'COMPARISON_GROUP_COL': 'comparison_group_1',
        'OUTCOME_EVENT_COL': 'pc_148_filed',
        'GIVEN_EVENT_COL': 'booked_148_only',
    },
    labels=LABELS,
)
pc148_rr_sum = Relative_risk.Ratio(
    df=sfda,
    params={
        'TREAT_GROUP_COL': 'def_black',
        'CONTROL_GROUP_COL': 'def_white',
        'OUTCOME_EVENT_OP': 'faces',
        'OUTCOME_EVENT_COL': 'pc_148_filed',
        'GIVEN_EVENT_COL': 'booked_148_only',
    },
    labels=LABELS,
)

Chi_square.DEFAULTS

Chi_square.CENSUS_PARAMS

Chi_square.EQUAL_PARAMS

params = {
    'Census': {
        'OBSERVED_DICT': pc148_sum_gc,
        'CENSUS_DICT': {k:v for k,v in magic['county'].items() if k != 'total'},
        'NULL_PHRASE': f"the distribution by race of {
            LABELS['booked_148_only']} follows the distribution of the general population"
    },
    'Equal': {
        'OBSERVED_DICT': pc148_sum_gc,
        'NULL_PHRASE': f"the distribution by race of {
            LABELS['booked_148_only']} is equal across all groups"
    },
}